Udemy Data Science & Machine Learning Bootcamp
Udemy Data Science & Machine Learning Bootcamp</mark></center>
In [1]:
# For displaying the figures:
from IPython import display
**1) NumPy:-** ¶
In [1]:
import numpy as np
1.1) NumPy Arrays¶
In [5]:
# From Python 1-D List to a 1-D NumPy Array:
l1 = [1, 2, 3]
arr_1d = np.array(l1)
print(l1)
print(arr_1d)
print()
# From Python 2-D List to a 2-D NumPy Array:
l2 = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
arr_2d = np.array(l2)
print(l2)
print(arr_2d)
In [9]:
# A range of numbers in a Python List Vs a NumPy Array:
lr = list(range(0, 11, 2))
nr = np.arange(0, 11, 2)
print(lr)
print(nr)
In [10]:
# Generating a NumPy array with all ones or zeros:
print(np.ones(3))
print(np.ones((3, 3))) # Notice the dimensions as a Tuple
print(np.zeros(3))
print(np.zeros((3, 3)))
In [25]:
# Evenly separated numbers in a NumPy Array:
al = np.linspace(0, 20, 5)
an = np.array(al)
print(al)
print(type(al))
print(an)
print(type(an))
In [26]:
# A Singular matrix in NumPy:
np.eye(3)
Out[26]:
In [44]:
# Random Numbers Matrices using NumPy Arrays:
arn1 = np.random.rand(5)
arn2 = np.random.rand(2, 3)
arn3 = np.random.rand(1, 3, 2)
print(arn1)
print(arn2)
print(arn3)
In [40]:
# Random Integer within a certain range:
rn1 = np.random.randint(15)
rn2 = np.random.randint(2, 10)
rn3 = np.random.randint(2, 50, 10)
print(rn1)
print(rn2)
print(rn3)
In [48]:
# Reshape a 1-D Array to a 2-D Matrix:
arr1 = np.arange(9)
arr2 = arr1.reshape(3, 3)
print(arr1)
print(arr2)
In [51]:
# Get the Min, Max values and their Index locations from an NumPy Array:
arn1 = np.random.rand(5)
arn2 = np.random.rand(2, 3)
print(arn1)
print(arn2)
print()
print(arn1.min())
print(arn1.argmin())
print()
print(arn2.min())
print(arn2.argmax())
In [53]:
# Get the shape of a NumPy Array:
arn1 = np.random.rand(5)
arn2 = np.random.rand(2, 3)
print(arn1)
print(arn2)
print()
print(arn1.shape)
print(arn2.shape)
In [56]:
# To get the data type of elements in a NumPy Array:
arn1 = np.random.rand(2, 3)
arr1 = np.arange(9)
arr2 = arr1.reshape(3, 3)
print(arr1)
print(arr2)
print(arn1.dtype)
In [76]:
# Array Indexing and Slicing:
ar1 = np.arange(9)
ar2 = ar1.reshape(3, 3)
print(ar1)
print(ar2)
print()
print(ar1[2])
print(ar1[3:8])
print(ar2[0][1])
print(ar2[2][2])
print()
print(ar2[:2, 1:])
print(ar2[1, :])
print(ar2[:, 1])
In [91]:
# Conditional Seletctions:
ar = np.arange(1, 11)
print(ar)
bool_ar = ar > 4
print(bool_ar)
arc = ar[bool_ar]
print(arc)
print(ar[ar >= 6])
1.2) NumPy Operations¶
In [95]:
# Adding and Subtracting two or more NumPy Arrays elementwise:
ar1 = np.arange(1, 6)
ar2 = np.arange(6, 11)
print(ar1)
print(ar2)
ara = ar1 + ar2
ars = ar1 - ar2
print(ara)
print(ars)
In [99]:
# A single element gets added/subtracted from an entire NumPy array:
arr = np.arange(21, 31)
print(arr)
print(arr-10)
print(arr+5)
In [103]:
# Universal NumPy Array functions for Mathematical operations:
arr = np.arange(1, 11)
print(np.sqrt(arr))
print(np.exp(arr))
print(np.log(arr))
**2) Pandas:-** ¶
In [2]:
import pandas as pd
2.1) Pandas Series¶
In [6]:
# Converting a List, NumPy array, Dictionary to a Pandas Series:
labels = ['a','b','c'] # Labels
my_list = [10,20,30] # List
np_arr = np.array([10,20,30]) # NumPy Array
d = {'a':10,'b':20,'c':30} # Dictionary
# List to Series
s1 = pd.Series(my_list)
# List to Series with index
s2 = pd.Series(my_list, labels)
# NumPy array to Series with index
s3 = pd.Series(np_arr, labels)
# Dictionary to Series- Keys as index labels
s4 = pd.Series(d)
print(s1, '\n***')
print(s2, '\n***')
print(s3, '\n***')
print(s4, '\n***')
In [17]:
# Using Series Index to access data points:
# First Subject
students_marks_ce1010 = pd.Series([27, 23, 19, 29, 30], ['John', 'Tim', 'Steve', 'Mike', 'Kevin'])
# Second Subject
students_marks_ce5410 = pd.Series([23, 25, 22, 24, 30], ['John', 'Tim', 'Steve', 'Mike', 'Kevin'])
print(students_marks_ce1010, '\n')
print(students_marks_ce5410, '\n')
# Accessing Scores
print(students_marks_ce1010['Tim'],'\n')
# Merging Series based off Index
total_marks = students_marks_ce1010 + students_marks_ce5410
print('Total Marks from both the Courses:\n',total_marks)
2.2) Pandas DataFrames¶
In [3]:
from numpy.random import randn
np.random.seed(101)
In [122]:
# Creating a DataFrame:
df = pd.DataFrame(randn(5,4), index='A B C D E'.split(), columns='W X Y Z'.split())
df
Out[122]:
2.2.1) Accessing and Removing data:¶
In [123]:
# Single column
print(df['W'])
print(type(df['W']), '\n')
# Multiple columns
print(df[['W', 'X']], '\n')
# Removing rows and columns (use axis=1)
df1 = df.drop('E')
df2 = df.drop('Z', axis=1)
print(df.drop('E'), '\n')
print(df.drop('Z', axis=1), '\n')
print('df1: \n',df1, '\n')
print('df2: \n',df2, '\n')
print('df: \n',df)
df.drop('Z', axis=1, inplace=True)
print('Updated df: \n',df)
# Adding new Columns
df['Z1'] = df['W'] + df['Y']
df
Out[123]:
2.2.2) Indexing and Slicing using loc and iloc:¶
In [124]:
# Accessing Rows using loc
print(df.loc['A'], '\n') # Single Row
print(df.loc[['A', 'C']], '\n') # Two different Rows
print(df.loc['A':'C'], '\n') # Slicing of Rows
# Accessing Rows using iloc
print(df.iloc[0], '\n') # Single Row
print(df.iloc[[0, 2]], '\n') # Two different Rows
print(df.iloc[0:2], '\n') # Slicing of Rows
'''Interesting fact to note is that .loc[m:n] will go from m till n, including n.
However, .iloc[i:j] will go from row / column no. i till no. j-1, excluding j.'''
# A subset of DataFrame can also be obtained
print(df.loc[['A', 'C'], ['W', 'Y']], '\n')
print(df.iloc[0:3,0:2])
2.2.3) Conditional Selection:¶
In [125]:
# Booleans
print(df>0, '\n')
print(df[df>0], '\n')
# Booleans on a particular column
print(df['W'] < 0, '\n')
print(df[df['W'] < 0], '\n')
df_sub = df[df['W']>0]
print(df_sub, '\n')
# We can also use multiple conditionals
print(df[(df['W']>0) & (df['Y'] < 0)], '\n') # AND
print(df[(df['W']>0) | (df['Y'] < 0)], '\n') # OR
2.2.4) Resetting and Chaging the index:¶
In [126]:
# Reset the Index: To the default index 0, 1, 2,...
print('Current DataFrame: \n',df)
print('\n',df.reset_index())
print('\n',df)
df.reset_index(inplace=True)
print('\n',df)
# Set a particular column as index
labels = [1, 2, 3, 4, 5]
df.set_index('index', inplace=True)
print('\n',df)
2.2.5) Multi-index and Index Hierarchy:¶
In [127]:
# Index Levels
outside = ['G1','G1','G1','G2','G2','G2']
inside = [1,2,3,1,2,3]
hier_index = list(zip(outside,inside))
# Forming the MultiIndex first
hier_index = pd.MultiIndex.from_tuples(hier_index)
In [131]:
hier_index
Out[131]:
In [129]:
df = pd.DataFrame(np.random.randn(6,2),index=hier_index,columns=['A','B'])
df
Out[129]:
In [260]:
df.loc['G1']
Out[260]:
In [140]:
df.loc['G1']['A']
Out[140]:
In [136]:
df.loc['G2'].loc[1]
Out[136]:
In [265]:
df.index.names
Out[265]:
In [132]:
df.index.names = ['Group','Num']
df
Out[132]:
In [270]:
df.xs('G1')
Out[270]:
In [138]:
df.xs(['G1',1])
Out[138]:
In [273]:
df.xs(1,level='Num')
Out[273]:
2.2.6) Handling Missing Values:¶
In [165]:
df = pd.DataFrame({'A':[1,2,np.nan],
'B':[5,np.nan,np.nan],
'C':[1,2,3]})
df
Out[165]:
In [166]:
# Finding the Missing Values:
df.isna()
Out[166]:
In [167]:
# Dropping Missing values across rows:
df.dropna(inplace=False)
Out[167]:
In [168]:
# Dropping Missing values across columns:
df.dropna(axis = 1, inplace=False)
Out[168]:
In [169]:
# Dropping only if Missing values are above a threshold 'x':
df.dropna(thresh=2)
Out[169]:
In [170]:
# Filling in all Missing Values with a column parameter:
df['B'].fillna(df['B'].mean(), inplace=True)
df
Out[170]:
2.2.7) Groupby Method:¶
In [177]:
# Create dataframe
data = {'Company':['GOOGLE','GOOGLE','AMAZON','AMAZON','FB','FB'],
'Person':['Sam','Charlie','Amy','Vanessa','Carl','Sarah'],
'Sales':[200,120,340,124,243,350]}
dfg = pd.DataFrame(data)
dfg
Out[177]:
In [180]:
# Forming a grouby based off a particular column:
df_by_company = dfg.groupby('Company')
In [181]:
# Calling the aggregate functions:
df_by_company.mean()
Out[181]:
In [182]:
df_by_company.sum()
Out[182]:
In [191]:
df_by_company.describe().transpose()
Out[191]:
In [198]:
df_by_company.describe().transpose()['GOOGLE']
Out[198]:
2.2.8) Merging, Joining, and Concatenating:¶
In [199]:
# Creating DataFrames:
df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']},
index=[0, 1, 2, 3])
df2 = pd.DataFrame({'A': ['A4', 'A5', 'A6', 'A7'],
'B': ['B4', 'B5', 'B6', 'B7'],
'C': ['C4', 'C5', 'C6', 'C7'],
'D': ['D4', 'D5', 'D6', 'D7']},
index=[4, 5, 6, 7])
df3 = pd.DataFrame({'A': ['A8', 'A9', 'A10', 'A11'],
'B': ['B8', 'B9', 'B10', 'B11'],
'C': ['C8', 'C9', 'C10', 'C11'],
'D': ['D8', 'D9', 'D10', 'D11']},
index=[8, 9, 10, 11])
In [200]:
# Vertical Concatenation
pd.concat([df1, df2, df3])
Out[200]:
In [201]:
# Horizontal Concatenation
pd.concat([df1, df2, df3], axis=1)
Out[201]:
In [ ]:
left = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']})
In [35]:
pd.merge(left,right,how='inner',on='key')
Out[35]:
Or to show a more complicated example:
In [37]:
left = pd.DataFrame({'key1': ['K0', 'K0', 'K1', 'K2'],
'key2': ['K0', 'K1', 'K0', 'K1'],
'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3']})
right = pd.DataFrame({'key1': ['K0', 'K1', 'K1', 'K2'],
'key2': ['K0', 'K0', 'K0', 'K0'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']})
In [39]:
pd.merge(left, right, on=['key1', 'key2'])
Out[39]:
In [40]:
pd.merge(left, right, how='outer', on=['key1', 'key2'])
Out[40]:
In [41]:
pd.merge(left, right, how='right', on=['key1', 'key2'])
Out[41]:
In [42]:
pd.merge(left, right, how='left', on=['key1', 'key2'])
Out[42]:
In [46]:
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
'B': ['B0', 'B1', 'B2']},
index=['K0', 'K1', 'K2'])
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
'D': ['D0', 'D2', 'D3']},
index=['K0', 'K2', 'K3'])
In [47]:
left.join(right)
Out[47]:
In [48]:
left.join(right, how='outer')
Out[48]:
In [ ]:
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
'B': ['B0', 'B1', 'B2']},
index=['K0', 'K1', 'K2'])
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
'D': ['D0', 'D2', 'D3']},
index=['K0', 'K2', 'K3'])
left.join(right)
left.join(right, how='outer')
2.3) Pandas Operations¶
In [230]:
# Creating a DataFrame:
df = pd.DataFrame({'col1':[1,2,3,4],'col2':[444,555,666,444],'col3':['abc','def','ghi','xyz']})
df.head()
Out[230]:
In [255]:
# Basic information about the DataFrame:
df.info()
In [236]:
# The list of all columns labels
df.columns
Out[236]:
In [237]:
# The list of all row labels
df.index
Out[237]:
2.3.1) Unique Values:¶
In [205]:
# Finding unique values in a column of a DataFrame:
df['col1'].unique()
Out[205]:
In [207]:
# Finding the count of unique values in a column of a DataFrame:
df['col2'].nunique()
Out[207]:
In [209]:
# Finding the frequency of values in a column of a DataFrame:
df['col2'].value_counts()
Out[209]:
In [240]:
# Finding the Null values in the DataFrame as a boolean
df.isnull()
Out[240]:
2.3.2) Conditionals:¶
In [211]:
# Finding the values from a column satisfying a given condition:
df[df['col1']>1]
Out[211]:
In [216]:
# Select from DataFrame using criteria from multiple columns:
df[(df['col1']>1) & (df['col2']>450)]
Out[216]:
2.3.3) Applying functions:¶
In [219]:
# Applying a function on each value in a column:
def modulus(x):
return x%10
df['col2'].apply(modulus)
Out[219]:
In [220]:
df['col3'].apply(len)
Out[220]:
2.3.4) Deleting & Adding a column:¶
In [233]:
# Drop a column
df.drop('col1', axis=1, inplace=False)
Out[233]:
In [234]:
# Drop a row
df.drop(1, inplace=False)
Out[234]:
In [227]:
# Permanently delete a column
del df['col2']
df
Out[227]:
In [231]:
# Permanently add a column
df['col2'] = [444, 555, 666, 444]
df
Out[231]:
2.3.5) Sorting the values in a column:¶
In [239]:
df.sort_values(by='col2')
Out[239]:
2.3.6) Pivot in a DataFrame:¶
In [242]:
data = {'A':['foo','foo','foo','bar','bar','bar'],
'B':['one','one','two','two','one','one'],
'C':['x','y','x','y','x','y'],
'D':[1,3,2,5,4,1]}
df = pd.DataFrame(data)
df
Out[242]:
In [91]:
df.pivot_table(values='D',index=['A', 'B'],columns=['C'])
Out[91]:
2.4) Data Input and Output:¶
2.4.1) Input and Output a CSV file:¶
In [248]:
df1 = pd.read_csv('example')
df1
Out[248]:
In [251]:
df1.to_csv('example',index=False)
2.4.2) Input and Output an Excel file:¶
In [253]:
df2 = pd.read_excel('Excel_Sample.xlsx')
df2
Out[253]:
In [254]:
df2.to_excel('Excel_Sample.xlsx')
**3) Matplotlib:-** ¶
In [4]:
import matplotlib.pyplot as plt
%matplotlib inline
# %matplotlib notebook
3.1) Basic Matplotlib Command:¶
In [9]:
x = np.linspace(0, 5, 11)
y = x ** 2
plt.plot(x, y, 'r--o')
plt.xlabel('X Axis Title Here')
plt.ylabel('Y Axis Title Here')
plt.title('String Title Here');
In [10]:
# Multiple Subplots
# plt.subplot(nrows, ncols, plot_number)
plt.subplot(1,2,1)
plt.plot(x, y, 'r--') # More on color options later
plt.subplot(1,2,2)
plt.plot(y, x, 'g*-');
3.2) Matplotlib Object Oriented Method:¶
In [45]:
# Create Figure (empty canvas)
fig = plt.figure()
# Add set of axes to figure
axes = fig.add_axes([0.1, 0.1, 0.8, 0.8]) # left, bottom, width, height (range 0 to 1)
# Plot on that set of axes
axes.plot(x, y, 'b-*')
axes.set_xlabel('Set X Label') # Notice the use of set_ to begin methods
axes.set_ylabel('Set y Label')
axes.set_title('Set Title')
Out[45]:
In [53]:
# Creates blank canvas
fig = plt.figure()
axes1 = fig.add_axes([0.1, 0.1, 0.8, 0.8]) # main axes
axes2 = fig.add_axes([0.2, 0.5, 0.4, 0.3]) # inset axes
# Larger Figure Axes 1
axes1.plot(x, y, 'b-s')
axes1.set_xlabel('X_label_axes2')
axes1.set_ylabel('Y_label_axes2')
axes1.set_title('Axes 2 Title')
# Insert Figure Axes 2
axes2.plot(y, x, 'r-.')
axes2.set_xlabel('X_label_axes2')
axes2.set_ylabel('Y_label_axes2')
axes2.set_title('Axes 2 Title');
3.3) Matplotlib Subplots:¶
In [57]:
# We can create a tuple unpacking where the 'axes' is basically an array of all the axes
fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(14,4))
# We can now call on each of the axes and plot
axes[0].plot(x, x**2, '--', 'r', label='x**2')
axes[1].plot(x, x**3, 'b-*', label='x**3')
axes[2].plot(x, x**2-5*x+6, 'g-s', label='x**2-5x+6')
# Set the Title and axes labels
axes[0].set_title('Linear')
axes[1].set_title('Cubic')
axes[2].set_title('Quadratic')
axes[0].set_xlabel('x')
axes[0].set_ylabel('y')
axes[1].set_xlabel('x')
axes[1].set_ylabel('y')
axes[2].set_xlabel('x')
axes[2].set_ylabel('y')
for ax in axes:
ax.legend(loc=0) # loc=0 finds the best location for the legend
plt.tight_layout()
3.4) Filling in the plots¶
In [16]:
linear_data = np.array([1,2,3,4,5,6,7,8])
exponential_data = linear_data**2
plt.figure()
# plot the linear data and the exponential data
plt.plot(linear_data, '-o', exponential_data, '-o')
# fill the area between the linear data and exponential data
plt.gca().fill_between(range(len(linear_data)),
linear_data, exponential_data,
facecolor='blue',
alpha=0.25);
3.5) Plotting with Pandas:¶
In [22]:
# # Seaborn comes with built-in data sets!
df = sns.load_dataset('tips')
df.head()
Out[22]:
In [23]:
df.plot.box();
In [25]:
df.plot.hist();
3.6) Figure Size and DPI:¶
In [307]:
fig = plt.figure(figsize=(5, 4),dpi=80)
ax = fig.add_axes([0, 0, 1, 1])
ax.plot(x, x**2)
ax.set_title('Quadratic')
ax.set_xlabel('x')
ax.set_ylabel('y');
3.6) Custom Plot Range:¶
In [59]:
fig, axes = plt.subplots(1, 3, figsize=(12, 4))
axes[0].plot(x, x**2, x, x**3)
axes[0].set_title("default axes ranges")
axes[1].plot(x, x**2, x, x**3)
axes[1].axis('tight')
axes[1].set_title("tight axes")
axes[2].plot(x, x**2, x, x**3)
axes[2].set_ylim([0, 60])
axes[2].set_xlim([2, 5])
axes[2].set_title("custom axes range");
plt.tight_layout()
**4) Seaborn:-** ¶
In [5]:
import seaborn as sns
sns.set_style('darkgrid')
In [12]:
# # Seaborn comes with built-in data sets!
tips = sns.load_dataset('tips')
tips.head()
Out[12]:
4.1) Distribution Plots:¶
4.1.1) Distplot:¶
In [13]:
%%timeit -n 100 # IPython Magic function
sns.distplot(tips['total_bill'], kde=False);
In [323]:
# We can also vary the bins size to get a more detailed distribution
sns.distplot(tips['total_bill'], kde=False, bins=40);
4.1.2) Jointplot:¶
In [14]:
# The default scatter kind plot
sns.jointplot(x='total_bill',y='tip',data=tips);
In [328]:
# The hexagonal plot
sns.jointplot(x='total_bill',y='tip',data=tips, kind='hex');
In [329]:
# The kde kind plot
sns.jointplot(x='total_bill',y='tip',data=tips, kind='kde');
In [330]:
# The regression plot
sns.jointplot(x='total_bill',y='tip',data=tips, kind='reg');
In [331]:
# The resid kind plot
sns.jointplot(x='total_bill',y='tip',data=tips, kind='resid');
4.1.3) Pairplot:¶
In [332]:
sns.pairplot(tips);
Out[332]:
In [333]:
sns.pairplot(tips, hue='sex');
4.1.4) Rugplot:¶
In [338]:
sns.rugplot(tips['total_bill']);
4.2) Categorical Plots:¶
4.2.1) Barplot and Countplot:¶
In [339]:
# Barplot with a Categorical variable on the x-axis and a numerical variable on the y-axis compaing means
sns.barplot(x='sex', y='total_bill', data=tips);
In [342]:
# Barplot with a Categorical variable on the x-axis and a numerical variable on the y-axis comparing std dev
sns.barplot(x='sex', y='total_bill', data=tips, estimator=np.std);
In [343]:
# Barplot with a Categorical variable on the x-axis and a numerical variable on the y-axis comparing median
sns.barplot(x='sex', y='total_bill', data=tips, estimator=np.median);
In [345]:
# Counplot for categorical variables
sns.countplot(tips['sex']);
In [347]:
# Counplot for categorical variables with hue
sns.countplot(tips['size'], hue=tips['sex']);
4.2.2) Boxplot and Violinplot:¶
In [349]:
# A boxplot with categorical variable on the x-axis and numerical varaiable on the y-axis
sns.boxplot(x='day', y='total_bill', data=tips);
In [350]:
# A boxplot with categorical variable on the x-axis and numerical varaiable on the y-axis with hue on sex
sns.boxplot(x='day', y='total_bill', data=tips, hue='sex');
In [352]:
# A boxplot with categorical variable on the x-axis and numerical varaiable on the y-axis with hue on smoker
sns.boxplot(x='day', y='total_bill', data=tips, hue='smoker');
In [353]:
# A violinplot with exact same arguments as the boxplot
sns.violinplot(x='day', y='total_bill', data=tips, hue='smoker');
In [356]:
# We can also merge these two hues with split
sns.violinplot(x='day', y='total_bill', data=tips, hue='smoker', split=True);
4.2.3) Factor Plots:¶
In [358]:
sns.factorplot(x='day', y='total_bill', data=tips, kind='bar');
4.3) Matrix Plots:¶
In [360]:
flights = sns.load_dataset('flights')
In [361]:
flights.head()
Out[361]:
4.3.1) Heatmaps:¶
In [388]:
sns.heatmap(tips.corr(), annot=True, cmap='magma', lw=1, linecolor='black');
In [380]:
sns.heatmap(flights.corr(), annot=True, cmap='RdBu_r');
4.4) Grid Plots:¶
In [389]:
iris = sns.load_dataset('iris')
iris.head()
Out[389]:
4.4.1) PairGrid:¶
In [396]:
# First create a PairGrid
g = sns.PairGrid(iris)
# Now plot or map on this PairGrid at upper,lower, and diagonal locations
g.map_diag(sns.distplot); # On the diagonal
g.map_upper(plt.scatter); # On the upper triangle
g.map_lower(sns.kdeplot); # On the lower triangle
4.4.2) FacetGrid:¶
In [412]:
# Create an empty FacetGrid
fg = sns.FacetGrid(tips, col='time', row='sex')
# Now map or plot to this grid with an uniariate plot
fg.map(sns.distplot, 'total_bill');
In [415]:
# Create an empty FacetGrid
fg1 = sns.FacetGrid(tips, col='time', row='sex')
# Now map or plot to this grid with a bivariate plot
fg1.map(sns.scatterplot, 'total_bill', 'tip');
4.5) Regression Plot:¶
In [416]:
sns.lmplot(x='total_bill', y='tip', data=tips);
In [418]:
# We can include a hue as well
sns.lmplot(x='total_bill', y='tip', data=tips, hue='sex', markers=['o', 'v']);
Using a Grid¶
In [419]:
sns.lmplot(x='total_bill',y='tip',data=tips,col='sex');
In [454]:
# We just need to include the column and rows (no need for any grid)
sns.lmplot(x="total_bill", y="tip", row="sex", col="time",data=tips);
In [524]:
# We can control the height and the width of the plots using aspect and height arguments
sns.lmplot(x='total_bill',y='tip',data=tips,col='day',hue='sex',palette='seismic', aspect=0.8, height=5);
4.6) Seaborn Style and Color:¶
4.6.1) Plot Style and Spines:¶
In [473]:
sns.set_style('white')
sns.countplot(tips['sex']);
sns.despine()
4.6.2) Size and Aspect¶
In [509]:
# Non Grid Plot
plt.figure(figsize=(6,4))
sns.countplot(x='sex',data=tips);
In [508]:
# Grid Type Plot
sns.lmplot(x='total_bill',y='tip',height=5,aspect=0.8,data=tips);
4.6.3) Scale and Context¶
In [16]:
print('We can get more colormaps and pallettes from:')
display.Image("./Matplotlib Colormaps.png", width=500, height=50)
Out[16]:
In [518]:
sns.set_context('notebook', font_scale=1);
sns.countplot(x='sex',data=tips,palette='seismic');
In [6]:
sns.set_style('whitegrid');
sns.set_context('notebook', font_scale=1);
sns.set_palette("RdBu_r")
**5) Machine Learning with Python:-** ¶
The main libraries to be used for Machine Learning in Python is the 'Scikit-learn'. The Algorithmic Cheat sheet is as displayed below:
**6) Linear Regression:-** ¶
In [7]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.linear_model import LinearRegression # The Linear Regression Model
from sklearn import metrics # For Regression Evaluation Metrics
In [17]:
# Importing the dataset for predicting the Price of a house:
df = pd.read_csv('USA_Housing-Copy1.csv')
In [19]:
df.head()
Out[19]:
In [20]:
df.info()
In [21]:
df.describe().transpose()
Out[21]:
6.1) EDA:¶
In [34]:
sns.pairplot(df);
In [42]:
fig, axes = plt.subplots(1, 5, figsize=(16, 4))
sns.distplot(df['Avg. Area Income'], ax=axes[0]);
sns.distplot(df['Avg. Area House Age'], ax=axes[1]);
sns.distplot(df['Avg. Area Number of Rooms'], ax=axes[2]);
sns.distplot(df['Avg. Area Number of Bedrooms'], ax=axes[3]);
sns.distplot(df['Area Population'], ax=axes[4]);
plt.tight_layout()
In [50]:
fig, axes = plt.subplots(1, 5, figsize=(14, 4))
sns.boxplot(df['Avg. Area Income'], ax=axes[0], orient='v');
sns.boxplot(df['Avg. Area House Age'], ax=axes[1], orient='v');
sns.boxplot(df['Avg. Area Number of Rooms'], ax=axes[2], orient='v');
sns.boxplot(df['Avg. Area Number of Bedrooms'], ax=axes[3], orient='v');
sns.boxplot(df['Area Population'], ax=axes[4], orient='v');
plt.tight_layout()
In [54]:
sns.heatmap(df.corr(), cmap='magma', annot=True);
6.2) Training our Linear Regression Model¶
In [84]:
# Columns having Numerical data are
numeric_features = df.select_dtypes(include=[np.number])
print(list(numeric_features.columns))
In [60]:
# Creating our X and y arrays yb separating the target labels column:
X = df[['Avg. Area Income', 'Avg. Area House Age', 'Avg. Area Number of Rooms',
'Avg. Area Number of Bedrooms', 'Area Population']]
y = df['Price']
6.2.1) Train Test split:¶
In [61]:
# The order of the split variables is important
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=101)
6.2.2) Creating and Training the Model¶
In [62]:
lm = LinearRegression()
lm.fit(X_train, y_train) # Fit the model on the train split
Out[62]:
6.2.3) Model Evaluation¶
In [63]:
# The intercept of the linear model:
lm.intercept_
Out[63]:
In [65]:
# For coefficients can be properly shown in a DataFrame:
# print(lm.coef_)
coeff_df = pd.DataFrame(lm.coef_,X.columns,columns=['Coefficient'])
coeff_df
Out[65]:
Interpreting the coefficients:
- Holding all other features fixed, a 1 unit increase in Avg. Area Income is associated with an increase of \$21.52 .
- Holding all other features fixed, a 1 unit increase in Avg. Area House Age is associated with an increase of \$164883.28 .
- Holding all other features fixed, a 1 unit increase in Avg. Area Number of Rooms is associated with an increase of \$122368.67 .
- Holding all other features fixed, a 1 unit increase in Avg. Area Number of Bedrooms is associated with an increase of \$2233.80 .
- Holding all other features fixed, a 1 unit increase in Area Population is associated with an increase of \$15.15 .
Does this make sense? Probably not because I made up this data. If you want real data to repeat this sort of analysis, check out the boston dataset:
from sklearn.datasets import load_boston
boston = load_boston()
print(boston.DESCR)
boston_df = boston.data6.2.4) Predictions from our Model¶
In [67]:
# Let's fetch the y_test lavels:
predictions = lm.predict(X_test)
predictions
In [78]:
sns.jointplot(y_test, predictions, kind='scatter');
Residual Histogram
In [71]:
# A close to normal distribution of Residuals indicates correctness of the model
sns.distplot((y_test-predictions), bins=50);
6.4) Regression Evaluation Metrics¶
Here are three common evaluation metrics for regression problems:
Mean Absolute Error (MAE) is the mean of the absolute value of the errors:
$$\frac 1n\sum_{i=1}^n|y_i-\hat{y}_i|$$Mean Squared Error (MSE) is the mean of the squared errors:
$$\frac 1n\sum_{i=1}^n(y_i-\hat{y}_i)^2$$Root Mean Squared Error (RMSE) is the square root of the mean of the squared errors:
$$\sqrt{\frac 1n\sum_{i=1}^n(y_i-\hat{y}_i)^2}$$Comparing these metrics:
- MAE is the easiest to understand, because it's the average error.
- MSE is more popular than MAE, because MSE "punishes" larger errors, which tends to be useful in the real world.
- RMSE is even more popular than MSE, because RMSE is interpretable in the "y" units.
All of these are loss functions, because we want to minimize them.
In [80]:
print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))
In [87]:
# The R^2 value can also be obtained as:
print('R^2 score: ', metrics.explained_variance_score(y_test, predictions))
**7) Logistic Regression:-** ¶
In [64]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.linear_model import LogisticRegression # The Linear Regression Model
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [241]:
# Importing the dataset for predicting the Price of a house:
df = pd.read_csv('titanic_train-Copy1.csv')
In [242]:
df.head()
Out[242]:
In [243]:
df.info()
In [244]:
df.describe().transpose()
Out[244]:
In [245]:
df.nunique()
Out[245]:
7.1) Handling Missing Data¶
In [246]:
dfn = df.isnull().describe().transpose()
dfn[dfn['unique']==2]
Out[246]:
In [247]:
# We can also use heatmaps to identify missing data
sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='seismic');
In [248]:
plt.figure(figsize=(12, 7))
sns.boxplot(x='Pclass',y='Age',data=df,palette='winter');
In [249]:
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age
In [250]:
df['Age'] = df[['Age','Pclass']].apply(impute_age,axis=1)
In [251]:
sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='seismic');
In [252]:
df.drop('Cabin',axis=1,inplace=True)
In [253]:
df.head()
Out[253]:
In [254]:
df.dropna(inplace=True)
In [255]:
sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='seismic');
7.2) EDA:¶
In [256]:
# Columns having Numerical data are
numeric_features = df.select_dtypes(include=[np.number])
print(list(numeric_features.columns))
In [257]:
sns.pairplot(df);
In [258]:
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
sns.distplot(df['Age'], ax=axes[0]);
sns.distplot(df['Fare'], ax=axes[1]);
plt.tight_layout()
In [259]:
fig, axes = plt.subplots(2, 3, figsize=(12, 6))
sns.countplot(df['Survived'], ax=axes[0][0]);
sns.countplot(df['Pclass'], ax=axes[0][1]);
sns.countplot(df['Sex'], ax=axes[0][2]);
sns.countplot(df['SibSp'], ax=axes[1][0]);
sns.countplot(df['Parch'], ax=axes[1][1]);
sns.countplot(df['Embarked'], ax=axes[1][2]);
plt.tight_layout()
In [260]:
fig, axes = plt.subplots(1, 3, figsize=(12, 4))
sns.boxplot(df['Age'], ax=axes[0], hue=df['Survived'], orient='v');
sns.countplot(df['Sex'], ax=axes[1], hue=df['Survived']);
sns.countplot(df['Pclass'], ax=axes[2], hue=df['Survived']);
plt.tight_layout()
7.3) Converting Categorical Features (Dummy Encoding)¶
In [261]:
df.info()
In [262]:
gender = pd.get_dummies(df['Sex'], drop_first=True)
embark = pd.get_dummies(df['Embarked'], drop_first=True)
# pclass = pd.get_dummies(df['Pclass'], drop_first=True)
In [263]:
df.drop(['Sex', 'Embarked','Name','Ticket'], axis=1, inplace=True)
In [264]:
df = pd.concat([df, gender, embark], axis=1)
In [265]:
df.drop('PassengerId', axis=1, inplace=True)
In [266]:
df.head()
Out[266]:
7.3) Building a Logistic Regression model¶
7.3.1) Train Test Split¶
In [267]:
X_train, X_test, y_train, y_test = train_test_split(df.drop('Survived',axis=1), df['Survived'], test_size=0.30, random_state=101)
7.3.2) Training and Predicting¶
In [268]:
logmodel = LogisticRegression()
logmodel.fit(X_train, y_train);
In [269]:
predictions = logmodel.predict(X_test);
7.3.3) Evaluation¶
In [270]:
print(classification_report(y_test,predictions))
In [271]:
print('The Confusion Matrix is: \n')
matrixc = confusion_matrix(y_test, predictions)
print(matrixc)
matrix = plot_confusion_matrix(logmodel ,X_test, y_test, values_format='d')
matrix.ax_.set_title('Confusion Matrix')
plt.xlabel('Predicted Label')
plt.gcf().axes[0].tick_params(color='black')
plt.gcf().set_size_inches(4,3)
plt.show()
**8) Plotly and Cufflinks:-** ¶
In [272]:
# Importing the required libraries:
import cufflinks as cf
cf.go_offline()
In [ ]:
In [ ]:
In [ ]:
**9) K-Nearest Neightbours:-** ¶
In [16]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.preprocessing import StandardScaler # For scaling the dataset
from sklearn.neighbors import KNeighborsClassifier # The KNN Model
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [27]:
df = pd.read_csv('KNN_Project_Data-Copy1')
In [28]:
df.head()
Out[28]:
In [29]:
df.info()
In [30]:
df.describe().transpose()
Out[30]:
In [31]:
df.nunique()
Out[31]:
In [32]:
lc = list(df.columns)
In [33]:
fig, axes = plt.subplots(2, 5, figsize=(16, 4))
j = 0
c = 0
for i in range(len(lc)-1):
sns.distplot(df[lc[i]], ax=axes[c][j]);
j += 1
if i == 4:
c, j = 1, 0
plt.tight_layout()
9.1) Standard Scaler on the variables:¶
In [34]:
# Create a StandardScaler:
scaler = StandardScaler()
In [35]:
# Fit the Scaler to the dataset excluding the labels:
scaler.fit(df.drop('TARGET CLASS', axis=1))
Out[35]:
In [36]:
# Scale or standardise the dataset:
scaled = scaler.transform(df.drop('TARGET CLASS', axis=1))
In [37]:
# Recreate the DataFrame
df1 = pd.DataFrame(scaled, columns = df.columns[:-1])
df1.head()
Out[37]:
9.2) Training the KNN Model:¶
In [38]:
X_train, X_test, y_train, y_test = train_test_split(df1,df['TARGET CLASS'], test_size=0.30)
In [39]:
knn = KNeighborsClassifier()
In [40]:
knn.fit(X_train, y_train)
Out[40]:
9.3) Predictions and Model Evaluation:¶
In [41]:
predictions = knn.predict(X_test)
In [42]:
print(classification_report(y_test,predictions))
In [43]:
print('The Confusion Matrix is: \n')
matrixc = confusion_matrix(y_test, predictions)
print(matrixc)
matrix = plot_confusion_matrix(knn ,X_test, y_test, values_format='d')
matrix.ax_.set_title('Confusion Matrix')
plt.xlabel('Predicted Label')
plt.gcf().axes[0].tick_params(color='black')
plt.gcf().set_size_inches(4,3)
plt.show()
9.4) Choosing a better K value:¶
In [44]:
error_rate = []
d = dict()
for i in range(1, 51):
knn_c = KNeighborsClassifier(n_neighbors=i)
knn_c.fit(X_train, y_train)
pred_i = knn_c.predict(X_test)
error_rate.append(np.mean(pred_i != y_test))
d[i] = np.mean(pred_i != y_test)
In [368]:
sns.jointplot(range(1, 51), error_rate, size=5);
In [45]:
mn = min(d.values())
for i in d:
if d[i] == mn:
print(i,':', mn)
9.5) Comparing improved performance with K value:¶
In [46]:
# KNN With K=1
knn = KNeighborsClassifier(n_neighbors=1)
knn.fit(X_train,y_train)
pred = knn.predict(X_test)
print('KNN with K=1:')
print('\n')
print(confusion_matrix(y_test,pred))
print('\n')
print(classification_report(y_test,pred))
print('*******************************************************')
# KNN With K=31
knn = KNeighborsClassifier(n_neighbors=31)
knn.fit(X_train,y_train)
pred = knn.predict(X_test)
print('KNN with K=31:')
print('\n')
print(confusion_matrix(y_test,pred))
print('\n')
print(classification_report(y_test,pred))
**10) Decision Trees and Random Forest:-** ¶
In [371]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.tree import DecisionTreeClassifier # For Decision Trees
from sklearn.ensemble import RandomForestClassifier # For Random Forest
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [372]:
df = pd.read_csv('kyphosis-Copy1.csv')
df.head()
Out[372]:
In [373]:
df.info()
In [374]:
df.describe().transpose()
Out[374]:
In [375]:
# Columns having Numerical data are
numeric_features = df.select_dtypes(include=[np.number])
print(list(numeric_features.columns))
In [376]:
sns.pairplot(df, hue='Kyphosis');
In [377]:
df.nunique()
Out[377]:
In [378]:
X = df[['Age', 'Number', 'Start']]
y = df['Kyphosis']
10.1) Training the Decision Tree Model¶
In [379]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30)
In [380]:
dtree = DecisionTreeClassifier()
In [381]:
dtree.fit(X_train, y_train)
Out[381]:
In [382]:
predictions = dtree.predict(X_test)
In [385]:
print(classification_report(y_test,predictions))
print('The Confusion Matrix is: \n')
matrixc = confusion_matrix(y_test, predictions)
print(matrixc)
10.2) Training the Random Forest Model¶
In [393]:
rfc = RandomForestClassifier(n_estimators=100)
In [394]:
rfc.fit(X_train, y_train)
Out[394]:
In [396]:
rfc_predictions = rfc.predict(X_test)
In [397]:
print(classification_report(y_test, rfc_predictions))
print('The Confusion Matrix is: \n')
matrixc = confusion_matrix(y_test, rfc_predictions)
print(matrixc)
**11) Support Vector Machines (SVM):-** ¶
In [65]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.svm import SVC # The Support Vector Classifier
from sklearn.model_selection import GridSearchCV # For Grid Search
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [2]:
from sklearn.datasets import load_breast_cancer
In [3]:
cancer = load_breast_cancer()
In [4]:
print(cancer['DESCR'])
In [13]:
# The main features DataFrame
df = pd.DataFrame(cancer['data'],columns=cancer['feature_names'])
df.head()
Out[13]:
In [17]:
# The prediction labels DataFrame
df_target = pd.DataFrame(cancer['target'],columns=['Cancer'])
In [14]:
df.info()
In [15]:
df.describe().transpose()
Out[15]:
In [ ]:
# sns.pairplot(df);
11.1) Training the SVC Model¶
In [21]:
X_train, X_test, y_train, y_test = train_test_split(df, cancer['target'], test_size=0.30, random_state=101)
In [24]:
svcmodel = SVC()
In [25]:
svcmodel.fit(X_train, y_train)
Out[25]:
In [26]:
predictions = svc.predict(X_test)
In [27]:
print(classification_report(y_test,predictions))
print('The Confusion Matrix is: \n')
matrixc = confusion_matrix(y_test, predictions)
print(matrixc)
11.2) Gridsearch¶
In [29]:
param_grid = {'C': [0.1,1, 10, 100, 1000], 'gamma': [1,0.1,0.01,0.001,0.0001], 'kernel': ['rbf']}
In [30]:
from sklearn.model_selection import GridSearchCV
In [31]:
grid = GridSearchCV(SVC(),param_grid,refit=True,verbose=3)
In [32]:
# May take a while!
grid.fit(X_train,y_train)
Out[32]:
In [33]:
grid.best_params_
Out[33]:
In [34]:
grid.best_estimator_
Out[34]:
In [35]:
grid_predictions = grid.predict(X_test)
In [36]:
print(confusion_matrix(y_test,grid_predictions))
In [37]:
print(classification_report(y_test,grid_predictions))
**12) K Means Clustering:-** ¶
In [39]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.cluster import KMeans # For KMeans Clustering
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [38]:
from sklearn.datasets import make_blobs
In [51]:
# Creating a Random Clustered dataset
data = make_blobs(n_samples=2000, n_features=2, centers=4, cluster_std=1.8,random_state=101)
12.1) EDA:¶
In [52]:
plt.scatter(data[0][:,0],data[0][:,1],c=data[1],cmap='rainbow');
12.2) Creating the Clusters Model:¶
In [53]:
kmeans = KMeans(n_clusters=4)
In [54]:
# Fit the Model to the features
kmeans.fit(data[0])
Out[54]:
In [55]:
kmeans.cluster_centers_
Out[55]:
In [56]:
kmeans.labels_
Out[56]:
In [57]:
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True,figsize=(10,6))
ax1.set_title('K Means')
ax1.scatter(data[0][:,0],data[0][:,1],c=kmeans.labels_,cmap='rainbow');
ax2.set_title("Original")
ax2.scatter(data[0][:,0],data[0][:,1],c=data[1],cmap='rainbow');
**13) Principal Component Analysis:-** ¶
In [ ]:
## <font color='darkred'> **12) K Means Clustering:-** <a name="p12"></a> </font>
In [66]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.preprocessing import StandardScaler # For scaling the data
from sklearn.decomposition import PCA # For PCA
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [68]:
from sklearn.datasets import load_breast_cancer
In [69]:
cancer = load_breast_cancer()
In [70]:
cancer.keys()
Out[70]:
In [71]:
print(cancer['DESCR'])
In [72]:
df = pd.DataFrame(cancer['data'],columns=cancer['feature_names'])
#(['DESCR', 'data', 'feature_names', 'target_names', 'target'])
In [73]:
df.head()
Out[73]:
In [74]:
# Scaling the data
scaler = StandardScaler()
scaler.fit(df)
Out[74]:
In [75]:
scaled_data = scaler.transform(df)
13.1) PCA Model:¶
In [76]:
pca = PCA(n_components=2)
In [77]:
pca.fit(scaled_data)
Out[77]:
Now we can transform this data to its first 2 principal components.
In [78]:
x_pca = pca.transform(scaled_data)
In [79]:
scaled_data.shape
Out[79]:
In [80]:
x_pca.shape
Out[80]:
Great! We've reduced 30 dimensions to just 2! Let's plot these two dimensions out!
In [81]:
plt.figure(figsize=(8,6))
plt.scatter(x_pca[:,0],x_pca[:,1],c=cancer['target'],cmap='plasma');
plt.xlabel('First principal component');
plt.ylabel('Second Principal Component');
13.2) Interpreting the components¶
Clearly by using these two components we can easily separate these two classes.
Unfortunately, with this great power of dimensionality reduction, comes the cost of being able to easily understand what these components represent.
The components correspond to combinations of the original features, the components themselves are stored as an attribute of the fitted PCA object:
In [82]:
pca.components_
Out[82]:
In [83]:
df_comp = pd.DataFrame(pca.components_,columns=cancer['feature_names'])
In [84]:
plt.figure(figsize=(12,6))
sns.heatmap(df_comp,cmap='plasma',);
**14) Recommender Systems:-** ¶
In [39]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.cluster import KMeans # For KMeans Clustering
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [86]:
# Importing the dataset
column_names = ['user_id', 'item_id', 'rating', 'timestamp']
df = pd.read_csv('u-Copy1.data', sep='\t', names=column_names)
In [91]:
# Getting the Movie Titles as well:
movie_titles = pd.read_csv("Movie_Id_Titles-Copy1")
movie_titles.head()
Out[91]:
In [ ]:
# Merging the DataFrames:
df = pd.merge(df,movie_titles,on='item_id')
In [93]:
df.head()
Out[93]:
In [94]:
df.info()
In [95]:
df.nunique()
Out[95]:
14.1) EDA¶
In [105]:
# Average Rating for each movie
meanr = pd.DataFrame(df.groupby('title').mean()['rating'])
meanr.sort_values(by='rating', ascending=False)
Out[105]:
In [107]:
# Number of Ratings for each movie
countr = pd.DataFrame(df.groupby('title').count()['rating'])
countr.sort_values(by='rating', ascending=False)
Out[107]:
In [110]:
dfr = pd.merge(meanr, countr, on='title')
dfr.rename(columns = {'rating_x': 'avg rating', 'rating_y': 'no of ratings'}, inplace = True)
dfr.head()
Out[110]:
In [115]:
sns.distplot(dfr['avg rating'], bins=50);
In [118]:
sns.jointplot(dfr['avg rating'], dfr['no of ratings']);
14.2) Recommender System Model:¶
In [119]:
moviemat = df.pivot_table(index='user_id',columns='title',values='rating')
moviemat.head()
Out[119]:
In [120]:
starwars_user_ratings = moviemat['Star Wars (1977)']
liarliar_user_ratings = moviemat['Liar Liar (1997)']
starwars_user_ratings.head()
Out[120]:
In [122]:
similar_to_starwars = moviemat.corrwith(starwars_user_ratings);
similar_to_liarliar = moviemat.corrwith(liarliar_user_ratings);
In [123]:
corr_starwars = pd.DataFrame(similar_to_starwars,columns=['Correlation'])
corr_starwars.dropna(inplace=True)
corr_starwars.head()
Out[123]:
In [124]:
corr_starwars.sort_values('Correlation',ascending=False).head(10)
Out[124]:
In [126]:
corr_starwars = corr_starwars.join(dfr['no of ratings'])
corr_starwars.head()
Out[126]:
In [128]:
corr_starwars[corr_starwars['no of ratings']>100].sort_values('Correlation',ascending=False).head()
Out[128]:
In [131]:
corr_liarliar = pd.DataFrame(similar_to_liarliar,columns=['Correlation'])
corr_liarliar.dropna(inplace=True)
corr_liarliar = corr_liarliar.join(dfr['no of ratings'])
corr_liarliar[corr_liarliar['no of ratings']>100].sort_values('Correlation',ascending=False).head()
Out[131]:
**15) Statistical Testing:-** ¶
In [15]:
# Importing new libraries from scipy
from scipy import stats
In [16]:
df = pd.read_csv('grades.csv')
In [18]:
df.head()
Out[18]:
In [19]:
df.describe().T
Out[19]:
In [20]:
df.info()
In [26]:
early_finishers = df[df['assignment1_submission'].apply(pd.to_datetime) < '2016']
early_finishers.head()
Out[26]:
In [24]:
late_finsihers = df[df['assignment1_submission'].apply(pd.to_datetime) > '2016']
late_finsihers.head()
Out[24]:
In [27]:
# Another solution. First, the dataframe df and the early_finishers share index values, so I really just
# want everything in the df which is not in early_finishers
late_finishers=df[~df.index.isin(early_finishers.index)]
late_finishers.head()
Out[27]:
In [28]:
early_finishers.describe().T
Out[28]:
In [29]:
late_finishers.describe().T
Out[29]:
In [30]:
# Let's bring in our ttest_ind function
from scipy.stats import ttest_ind
In [31]:
# Let's run this function with our two populations, looking at the assignment 1 grades
ttest_ind(early_finishers['assignment1_grade'], late_finishers['assignment1_grade'])
Out[31]:
In [32]:
# Checking the other assignment grades as well:
print(ttest_ind(early_finishers['assignment2_grade'], late_finishers['assignment2_grade']))
print(ttest_ind(early_finishers['assignment3_grade'], late_finishers['assignment3_grade']))
print(ttest_ind(early_finishers['assignment4_grade'], late_finishers['assignment4_grade']))
print(ttest_ind(early_finishers['assignment5_grade'], late_finishers['assignment5_grade']))
print(ttest_ind(early_finishers['assignment6_grade'], late_finishers['assignment6_grade']))
**16) Naive Bayes:-** ¶
In [17]:
# Importing the required libraries:
from sklearn.model_selection import train_test_split # For Train-Test split
from sklearn.preprocessing import StandardScaler # For scaling the dataset
from sklearn.naive_bayes import GaussianNB # The Gaussian Naive Bayes
from sklearn.metrics import classification_report # For Classification Report
from sklearn.metrics import plot_confusion_matrix # For Plotting Confusion Matrix
from sklearn.metrics import confusion_matrix # For Confusion Matrix
In [10]:
df = pd.read_csv('diabetes.csv')
In [11]:
df.head()
Out[11]:
In [12]:
df.info()
In [13]:
df.describe().T
Out[13]:
In [17]:
sns.pairplot(df);
In [15]:
X_train, X_test, y_train, y_test = train_test_split(df.drop('Outcome', axis=1), df['Outcome'], test_size=0.3, random_state=42)
In [18]:
nb = GaussianNB()
In [19]:
nb.fit(X_train, y_train)
Out[19]:
In [20]:
predictions = nb.predict(X_test)
In [25]:
nb.score(X_test, y_test)
Out[25]:
In [29]:
print(classification_report(y_test, predictions))
print(confusion_matrix(y_test, predictions))
**17) Time Series:-** ¶
In [106]:
# Importing the required libraries:
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
from pandas.tseries.offsets import Hour, Minute
PREVIOUS_MAX_ROWS = pd.options.display.max_rows
pd.options.display.max_rows = 20
np.set_printoptions(precision=4, suppress=True)
17.1) Date and Time Data Types and Tools¶
In [40]:
today = datetime.now()
today
Out[40]:
In [47]:
print(today.date())
print(today.day)
print(today.month)
print(today.year)
print(today.hour)
print(today.minute)
print(today.second)
print(today.astimezone())
print(today.timestamp())
In [56]:
delta = datetime(2011, 1, 7) - datetime(2008, 6, 24, 8, 15)
print(delta)
print(delta.days)
print(delta.seconds)
In [59]:
# Using timedelta
start = datetime(2011, 1, 7)
start + timedelta(12)
start - 2*timedelta(12)
Out[59]:
17.2) Converting Between String and Datetime¶
In [63]:
stamp = datetime(year=2020, month=6, day=28)
str(stamp)
Out[63]:
In [70]:
# Conversion from timestamp to string
s1 = stamp.strftime('%y-%m-%d')
s2 = stamp.strftime('%Y-%m-%d')
print(s1)
print(s2)
In [73]:
# Conversion from string to timestamp
value = '2020-06-28'
d1 = datetime.strptime(value, '%Y-%m-%d')
d1
Out[73]:
In [76]:
d2 = parse('Jan 31, 1997 10:45 PM')
d2
Out[76]:
In [77]:
d3 = parse('6/12/2011', dayfirst=True)
d3
Out[77]:
In [78]:
datestrs = ['2011-07-06 12:00:00', '2011-08-06 00:00:00']
pd.to_datetime(datestrs)
Out[78]:
17.3) Time Series Basics¶
In [85]:
dates = [datetime(2011, 1, 2), datetime(2011, 1, 5),
datetime(2011, 1, 7), datetime(2011, 1, 8),
datetime(2011, 1, 10), datetime(2011, 1, 12)]
ts = pd.Series(np.random.randn(6), index=dates)
ts.head()
Out[85]:
In [80]:
ts.index
Out[80]:
In [81]:
ts.index.dtype
Out[81]:
In [82]:
stamp = ts.index[0]
stamp
Out[82]:
17.4) Indexing, Selection, Subsetting¶
In [87]:
# We can use Timestamp indices in many different formats
print(ts['1/10/2011'])
print(ts['20110110'])
print(ts['2011-01-10'])
In [88]:
longer_ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
longer_ts
Out[88]:
In [89]:
# Selecting only the indices from May 2001
longer_ts['2001-05']
Out[89]:
In [90]:
ts['1/6/2011':'1/11/2011']
Out[90]:
17.5) Time Series with Duplicate Indices¶
In [91]:
dates = pd.DatetimeIndex(['1/1/2000', '1/2/2000', '1/2/2000',
'1/2/2000', '1/3/2000'])
dup_ts = pd.Series(np.arange(5), index=dates)
dup_ts
Out[91]:
In [92]:
# Checking if all the index timestamps are unique
dup_ts.index.is_unique
Out[92]:
In [93]:
print(dup_ts['1/3/2000']) # not duplicated
print()
print(dup_ts['1/2/2000']) # duplicated
In [94]:
# Using groupby for duplicate indices
grouped = dup_ts.groupby(level=0)
grouped.mean()
grouped.count()
Out[94]:
17.6) Date Ranges, Frequencies, and Shifting¶
In [98]:
ts
resampler = ts.resample('D') # 'D' indicates daily frequency
17.6.1) Generating Date Ranges¶
In [105]:
index = pd.date_range('2012-04-01', '2012-06-01')
index
Out[105]:
In [100]:
# From start date, 20 periods
pd.date_range(start='2012-04-01', periods=20)
Out[100]:
In [101]:
# Reverse from end date, 20 periods
pd.date_range(end='2012-06-01', periods=20)
Out[101]:
In [102]:
pd.date_range('2000-01-01', '2000-12-01', freq='BM')
Out[102]:
In [103]:
pd.date_range('2012-05-02 12:56:31', periods=5)
Out[103]:
In [104]:
pd.date_range('2012-05-02 12:56:31', periods=5, normalize=True)
Out[104]:
17.6.2) Frequencies and Date Offsets¶
In [107]:
pd.date_range('2000-01-01', '2000-01-03 23:59', freq='4h')
Out[107]:
In [108]:
pd.date_range('2000-01-01', periods=10, freq='1h30min')
Out[108]:
17.6.3) Shifting (Leading and Lagging) Data¶
In [109]:
ts = pd.Series(np.random.randn(4), index=pd.date_range('1/1/2000', periods=4, freq='M'))
print(ts)
print()
ts.shift(2)
ts.shift(-2)
Out[109]:
Time Zone Handling¶
In [110]:
import pytz
pytz.common_timezones[-5:]
Out[110]:
In [111]:
tz = pytz.timezone('America/New_York')
tz
Out[111]:
Time Zone Localization and Conversion¶
In [112]:
rng = pd.date_range('3/9/2012 9:30', periods=6, freq='D')
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ts
Out[112]:
In [113]:
print(ts.index.tz)
In [114]:
pd.date_range('3/9/2012 9:30', periods=10, freq='D', tz='UTC')
Out[114]:
In [115]:
ts
ts_utc = ts.tz_localize('UTC')
ts_utc
ts_utc.index
Out[115]:
In [116]:
ts_utc.tz_convert('America/New_York')
Out[116]:
In [117]:
ts_eastern = ts.tz_localize('America/New_York')
ts_eastern.tz_convert('UTC')
ts_eastern.tz_convert('Europe/Berlin')
Out[117]:
In [118]:
ts.index.tz_localize('Asia/Shanghai')
Out[118]:
Operations with Time Zone−Aware Timestamp Objects¶
In [119]:
stamp = pd.Timestamp('2011-03-12 04:00')
stamp_utc = stamp.tz_localize('utc')
stamp_utc.tz_convert('America/New_York')
Out[119]:
In [120]:
stamp_moscow = pd.Timestamp('2011-03-12 04:00', tz='Europe/Moscow')
stamp_moscow
Out[120]:
In [121]:
stamp_utc.value
stamp_utc.tz_convert('America/New_York').value
Out[121]:
In [122]:
from pandas.tseries.offsets import Hour
stamp = pd.Timestamp('2012-03-12 01:30', tz='US/Eastern')
stamp
stamp + Hour()
Out[122]:
In [123]:
stamp = pd.Timestamp('2012-11-04 00:30', tz='US/Eastern')
stamp
stamp + 2 * Hour()
Out[123]:
Operations Between Different Time Zones¶
In [124]:
rng = pd.date_range('3/7/2012 9:30', periods=10, freq='B')
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ts
ts1 = ts[:7].tz_localize('Europe/London')
ts2 = ts1[2:].tz_convert('Europe/Moscow')
result = ts1 + ts2
result.index
Out[124]:
Periods and Period Arithmetic¶
In [125]:
p = pd.Period(2007, freq='A-DEC')
p
Out[125]:
In [126]:
p + 5
p - 2
Out[126]:
In [127]:
pd.Period('2014', freq='A-DEC') - p
Out[127]:
In [128]:
rng = pd.period_range('2000-01-01', '2000-06-30', freq='M')
rng
Out[128]:
In [129]:
pd.Series(np.random.randn(6), index=rng)
Out[129]:
In [130]:
values = ['2001Q3', '2002Q2', '2003Q1']
index = pd.PeriodIndex(values, freq='Q-DEC')
index
Out[130]:
Period Frequency Conversion¶
In [131]:
p = pd.Period('2007', freq='A-DEC')
p
p.asfreq('M', how='start')
p.asfreq('M', how='end')
Out[131]:
In [132]:
p = pd.Period('2007', freq='A-JUN')
p
p.asfreq('M', 'start')
p.asfreq('M', 'end')
Out[132]:
In [133]:
p = pd.Period('Aug-2007', 'M')
p.asfreq('A-JUN')
Out[133]:
In [134]:
rng = pd.period_range('2006', '2009', freq='A-DEC')
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ts
ts.asfreq('M', how='start')
Out[134]:
In [135]:
ts.asfreq('B', how='end')
Out[135]:
Quarterly Period Frequencies¶
In [136]:
p = pd.Period('2012Q4', freq='Q-JAN')
p
Out[136]:
In [137]:
p.asfreq('D', 'start')
p.asfreq('D', 'end')
Out[137]:
In [138]:
p4pm = (p.asfreq('B', 'e') - 1).asfreq('T', 's') + 16 * 60
p4pm
p4pm.to_timestamp()
Out[138]:
In [296]:
rng = pd.period_range('2011Q3', '2012Q4', freq='Q-JAN')
ts = pd.Series(np.arange(len(rng)), index=rng)
ts
Out[296]:
Converting Timestamps to Periods (and Back)¶
In [140]:
rng = pd.date_range('2000-01-01', periods=3, freq='M')
ts = pd.Series(np.random.randn(3), index=rng)
ts
pts = ts.to_period()
pts
Out[140]:
In [141]:
rng = pd.date_range('1/29/2000', periods=6, freq='D')
ts2 = pd.Series(np.random.randn(6), index=rng)
ts2
ts2.to_period('M')
Out[141]:
In [142]:
pts = ts2.to_period()
pts
pts.to_timestamp(how='end')
Out[142]:
Creating a PeriodIndex from Arrays¶
In [145]:
data = pd.read_csv('macrodata.csv')
data.head(5)
data.year
data.quarter
Out[145]:
In [146]:
index = pd.PeriodIndex(year=data.year, quarter=data.quarter,
freq='Q-DEC')
index
data.index = index
data.infl
Out[146]:
Resampling and Frequency Conversion¶
In [147]:
rng = pd.date_range('2000-01-01', periods=100, freq='D')
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ts
ts.resample('M').mean()
ts.resample('M', kind='period').mean()
Out[147]:
Downsampling¶
In [148]:
rng = pd.date_range('2000-01-01', periods=12, freq='T')
ts = pd.Series(np.arange(12), index=rng)
ts
Out[148]:
In [149]:
ts.resample('5min', closed='right').sum()
Out[149]:
In [150]:
ts.resample('5min', closed='right').sum()
Out[150]:
In [151]:
ts.resample('5min', closed='right', label='right').sum()
Out[151]:
In [152]:
ts.resample('5min', closed='right',
label='right', loffset='-1s').sum()
Out[152]:
Open-High-Low-Close (OHLC) resampling¶
In [153]:
ts.resample('5min').ohlc()
Out[153]:
Upsampling and Interpolation¶
In [154]:
frame = pd.DataFrame(np.random.randn(2, 4),
index=pd.date_range('1/1/2000', periods=2,
freq='W-WED'),
columns=['Colorado', 'Texas', 'New York', 'Ohio'])
frame
Out[154]:
In [155]:
df_daily = frame.resample('D').asfreq()
df_daily
Out[155]:
In [156]:
frame.resample('D').ffill()
Out[156]:
In [157]:
frame.resample('D').ffill(limit=2)
Out[157]:
In [158]:
frame.resample('W-THU').ffill()
Out[158]:
Resampling with Periods¶
In [159]:
frame = pd.DataFrame(np.random.randn(24, 4),
index=pd.period_range('1-2000', '12-2001',
freq='M'),
columns=['Colorado', 'Texas', 'New York', 'Ohio'])
frame[:5]
annual_frame = frame.resample('A-DEC').mean()
annual_frame
Out[159]:
In [160]:
# Q-DEC: Quarterly, year ending in December
annual_frame.resample('Q-DEC').ffill()
annual_frame.resample('Q-DEC', convention='end').ffill()
Out[160]:
In [161]:
annual_frame.resample('Q-MAR').ffill()
Out[161]:
Moving Window Functions¶
In [162]:
close_px_all = pd.read_csv('stock_px_2.csv',
parse_dates=True, index_col=0)
close_px = close_px_all[['AAPL', 'MSFT', 'XOM']]
close_px = close_px.resample('B').ffill()
In [163]:
close_px.AAPL.plot()
close_px.AAPL.rolling(250).mean().plot()
Out[163]:
In [164]:
plt.figure()
Out[164]:
In [165]:
appl_std250 = close_px.AAPL.rolling(250, min_periods=10).std()
appl_std250[5:12]
appl_std250.plot()
Out[165]:
In [166]:
expanding_mean = appl_std250.expanding().mean()
In [167]:
plt.figure()
Out[167]:
In [168]:
close_px.rolling(60).mean().plot(logy=True)
Out[168]:
In [169]:
close_px.rolling('20D').mean()
Out[169]:
Exponentially Weighted Functions¶
In [170]:
plt.figure()
Out[170]:
In [171]:
aapl_px = close_px.AAPL['2006':'2007']
ma60 = aapl_px.rolling(30, min_periods=20).mean()
ewma60 = aapl_px.ewm(span=30).mean()
ma60.plot(style='k--', label='Simple MA')
ewma60.plot(style='k-', label='EW MA')
plt.legend()
Out[171]:
Binary Moving Window Functions¶
In [172]:
plt.figure()
Out[172]:
In [173]:
spx_px = close_px_all['SPX']
spx_rets = spx_px.pct_change()
returns = close_px.pct_change()
In [174]:
corr = returns.AAPL.rolling(125, min_periods=100).corr(spx_rets)
corr.plot()
Out[174]:
In [175]:
plt.figure()
Out[175]:
In [176]:
corr = returns.rolling(125, min_periods=100).corr(spx_rets)
corr.plot()
Out[176]:
User-Defined Moving Window Functions¶
In [177]:
plt.figure()
Out[177]:
In [178]:
from scipy.stats import percentileofscore
score_at_2percent = lambda x: percentileofscore(x, 0.02)
result = returns.AAPL.rolling(250).apply(score_at_2percent)
result.plot()
Out[178]:
17.7) Time Series Analysis Project 1¶
In [257]:
# Importing the dataset
df = pd.read_csv('AirPassengers.csv')
In [258]:
df
Out[258]:
In [259]:
df.info()
In [260]:
df.describe().T
Out[260]:
In [261]:
df.index
Out[261]:
In [262]:
# Parse strings to datetime type
df['Month'] = pd.to_datetime(df['Month'], infer_datetime_format=True) #convert from string to datetime
df = df.set_index('Month')
df
Out[262]:
In [263]:
df.columns
Out[263]:
17.7.1) EDA¶
In [264]:
# Checking the Stationarity
plt.plot(df);
In [265]:
#Test whether Timeseries is Stationary or not
from statsmodels.tsa.stattools import adfuller
def test_stationarity(timeseries):
#Determing rolling statistics
rolmean = timeseries.rolling(window=12).mean()
rolstd = timeseries.rolling(window=12).std()
#Plot rolling statistics:
print ('Rolling Statistics Test:')
orig = plt.plot(timeseries, color='blue',label='Original')
mean = plt.plot(rolmean, color='red', label='Rolling Mean')
std = plt.plot(rolstd, color='black', label = 'Rolling Std')
plt.legend(loc='best')
plt.title('Rolling Mean & Standard Deviation')
plt.show(block=False)
#Perform Augmented-Dickey-Fuller (ADF) test:
print ('Results of Augmented-Dickey-Fuller (ADF) Test:')
dftest = adfuller(timeseries['#Passengers'], autolag='AIC')
dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])
for key,value in dftest[4].items():
dfoutput['Critical Value (%s)'%key] = value
print (dfoutput)
if dfoutput.loc['p-value'] < 0.05:
print('\n>> Reject H0, the dataset can be stationary')
else:
print('\n>> Failed to Reject H0, the dataset can be non-stationary!')
test_stationarity(df)
17.7.2) Converting from Non-Stationary to Stationary dataset¶
Log method¶
In [270]:
df_log = np.log(df)
df_log.dropna(inplace=True)
test_stationarity(df_log)
Subtracting Simple Rolling Averages¶
In [271]:
moving_avg = df_log.rolling(window=12).mean()
df_moving_avg = df_log - moving_avg
df_moving_avg.dropna(inplace=True)
test_stationarity(df_moving_avg)
Subtracting Exponential Rolling Averages¶
In [273]:
exp_moving_avg = df_log.ewm(halflife=12, min_periods=0, adjust=True).mean()
df_exp_moving_avg = df_log - exp_moving_avg
df_exp_moving_avg.dropna(inplace=True)
test_stationarity(df_exp_moving_avg)
Subtracting Previous values shift()¶
In [274]:
df_log_shift = df_log - df_log.shift()
df_log_shift.dropna(inplace=True)
test_stationarity(df_log_shift)
17.8) Time Series Analysis Project 2¶
In [275]:
import statsmodels.api as sm
In [277]:
df = pd.read_excel("Sample - Superstore.xls")
furniture = df.loc[df['Category'] == 'Furniture']
In [278]:
df.describe()
Out[278]:
In [279]:
furniture['Order Date'].min(), furniture['Order Date'].max()
Out[279]:
17.8.1) Data Preprocessing¶
In [280]:
furniture.isnull().sum()
Out[280]:
In [281]:
furniture = furniture.groupby('Order Date')['Sales'].sum().reset_index()
In [282]:
# Indexing with time series data
furniture = furniture.set_index('Order Date')
furniture.index
Out[282]:
In [283]:
y = furniture['Sales'].resample('MS').mean()
In [286]:
y['2017':]
Out[286]:
In [287]:
y.plot(figsize=(15, 6))
plt.show()
In [288]:
from pylab import rcParams
rcParams['figure.figsize'] = 18, 8
decomposition = sm.tsa.seasonal_decompose(y, model='additive')
fig = decomposition.plot()
plt.show()
17.8.2) Time series forecasting with ARIMA¶
In [292]:
import itertools
p = d = q = range(0, 2)
pdq = list(itertools.product(p, d, q))
seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]
for param in pdq:
for param_seasonal in seasonal_pdq:
try:
mod = sm.tsa.statespace.SARIMAX(y,
order=param,
seasonal_order=param_seasonal,
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic));
except:
continue
17.8.3) Fitting the ARIMA model¶
In [293]:
mod = sm.tsa.statespace.SARIMAX(y,
order=(1, 1, 1),
seasonal_order=(1, 1, 0, 12),
enforce_stationarity=False,
enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])
In [294]:
results.plot_diagnostics(figsize=(16, 8))
plt.show()
17.8.4) Validation¶
In [295]:
pred = results.get_prediction(start=pd.to_datetime('2017-01-01'), dynamic=False)
pred_ci = pred.conf_int()
ax = y['2014':].plot(label='observed')
pred.predicted_mean.plot(ax=ax, label='One-step ahead Forecast', alpha=.7, figsize=(14, 7))
ax.fill_between(pred_ci.index,
pred_ci.iloc[:, 0],
pred_ci.iloc[:, 1], color='k', alpha=.2)
ax.set_xlabel('Date')
ax.set_ylabel('Furniture Sales')
plt.legend()
plt.show()
**18) Natural Language Processing (NLP):-** ¶
In [180]:
# Importing the required libraries:
import nltk
In [181]:
# Importing the dataset
# nltk.download_shell()
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
**19) Deep Learning:-** ¶
In [24]:
import tensorflow as tf
19.1) TensorFlow:¶
In [9]:
df = pd.read_csv('fake_reg.csv')
df.head()
Out[9]:
In [11]:
df.describe().T
Out[11]:
In [12]:
sns.pairplot(df);
In [13]:
X = df[['feature1', 'feature2']].values
y = df['price']
In [15]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
In [18]:
sc = StandardScaler()
In [21]:
X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)
In [23]:
X_train.max()
Out[23]:
Building a Model¶
Choosing an optimizer and loss¶
Keep in mind what kind of problem you are trying to solve:
# For a multi-class classification problem
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# For a binary classification problem
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
# For a mean squared error regression problem
model.compile(optimizer='rmsprop',
loss='mse')In [31]:
## Building a Neural Network Model
model_ = tf.keras.Sequential([tf.keras.layers.Dense(units=4, activation='relu'),
tf.keras.layers.Dense(units=2, activation='relu'),
tf.keras.layers.Dense(units=1)]) # First layer with 4 Neurons, nest with 2 and the final Output layer.
# Another way to do build such a model is:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
model = Sequential()
# Add Layers now:
model.add(Dense(units=4, activation='relu')) # A Rectlinear Activation Function Layer with 4 Neurons
model.add(Dense(units=2, activation='relu')) # A Rectlinear Activation Function Layer with 4 Neurons
model.add(Dense(units=1)) # Final Output Layer
model.compile(optimizer='rmsprop', loss='mse')
Training¶
Below are some common definitions that are necessary to know and understand to correctly utilize Keras:
- Sample: one element of a dataset.
- Example: one image is a sample in a convolutional network
- Example: one audio file is a sample for a speech recognition model
- Batch: a set of N samples. The samples in a batch are processed independently, in parallel. If training, a batch results in only one update to the model.A batch generally approximates the distribution of the input data better than a single input. The larger the batch, the better the approximation; however, it is also true that the batch will take longer to process and will still result in only one update. For inference (evaluate/predict), it is recommended to pick a batch size that is as large as you can afford without going out of memory (since larger batches will usually result in faster evaluation/prediction).
- Epoch: an arbitrary cutoff, generally defined as "one pass over the entire dataset", used to separate training into distinct phases, which is useful for logging and periodic evaluation.
- When using validation_data or validation_split with the fit method of Keras models, evaluation will be run at the end of every epoch.
- Within Keras, there is the ability to add callbacks specifically designed to be run at the end of an epoch. Examples of these are learning rate changes and model checkpointing (saving).
In [32]:
model.fit(X_train,y_train,epochs=250)
Out[32]:
Evaluation¶
Let's evaluate our performance on our training set and our test set. We can compare these two performances to check for overfitting.
In [33]:
model.history.history
Out[33]:
In [34]:
loss = model.history.history['loss']
In [35]:
sns.lineplot(x=range(len(loss)),y=loss)
plt.title("Training Loss per Epoch");
Compare final evaluation (MSE) on training set and test set.¶
In [36]:
model.metrics_names
Out[36]:
In [38]:
# Loss on the Training data
model.evaluate(X_train, y_train)
Out[38]:
In [37]:
# Loss on the Test data
model.evaluate(X_test, y_test)
Out[37]:
In [40]:
test_predictions = model.predict(X_test)
test_predictions
Out[40]:
19.1) TensorFlow Regression:¶
The Data¶
We will be using data from a Kaggle data set:
https://www.kaggle.com/harlfoxem/housesalesprediction
Feature Columns¶
- id - Unique ID for each home sold
- date - Date of the home sale
- price - Price of each home sold
- bedrooms - Number of bedrooms
- bathrooms - Number of bathrooms, where .5 accounts for a room with a toilet but no shower
- sqft_living - Square footage of the apartments interior living space
- sqft_lot - Square footage of the land space
- floors - Number of floors
- waterfront - A dummy variable for whether the apartment was overlooking the waterfront or not
- view - An index from 0 to 4 of how good the view of the property was
- condition - An index from 1 to 5 on the condition of the apartment,
- grade - An index from 1 to 13, where 1-3 falls short of building construction and design, 7 has an average level of construction and design, and 11-13 have a high quality level of construction and design.
- sqft_above - The square footage of the interior housing space that is above ground level
- sqft_basement - The square footage of the interior housing space that is below ground level
- yr_built - The year the house was initially built
- yr_renovated - The year of the house’s last renovation
- zipcode - What zipcode area the house is in
- lat - Lattitude
- long - Longitude
- sqft_living15 - The square footage of interior housing living space for the nearest 15 neighbors
- sqft_lot15 - The square footage of the land lots of the nearest 15 neighbors
In [45]:
df = pd.read_csv('kc_house_data.csv')
df
Out[45]:
In [47]:
df.describe().T
Out[47]:
In [49]:
sns.distplot(df['price']);
In [50]:
sns.countplot(df['bedrooms']);
In [52]:
plt.figure(figsize=(12,8))
sns.scatterplot(x='long',y='lat',
data=df,hue='price',
palette='RdYlGn',edgecolor=None,alpha=0.2);
In [53]:
non_top_1_perc = df.sort_values('price',ascending=False).iloc[216:]
In [54]:
df.drop('id', axis=1, inplace=True)
In [55]:
df['date'] = pd.to_datetime(df['date'])
In [56]:
df['year'] = df['date'].apply(lambda date: date.year)
In [57]:
df['month'] = df['date'].apply(lambda date: date.month)
In [24]:
sns.boxplot(x='year',y='price',data=df)
Out[24]:
In [25]:
sns.boxplot(x='month',y='price',data=df)
Out[25]:
In [26]:
df.groupby('month').mean()['price'].plot()
Out[26]:
In [27]:
df.groupby('year').mean()['price'].plot()
Out[27]:
In [58]:
df = df.drop('date',axis=1)
In [59]:
df.columns
Out[59]:
In [60]:
# https://i.pinimg.com/originals/4a/ab/31/4aab31ce95d5b8474fd2cc063f334178.jpg
# May be worth considering to remove this or feature engineer categories from it
df['zipcode'].value_counts()
Out[60]:
In [61]:
df = df.drop('zipcode',axis=1)
In [62]:
df.head()
Out[62]:
In [63]:
# could make sense due to scaling, higher should correlate to more value
df['yr_renovated'].value_counts()
Out[63]:
In [64]:
df['sqft_basement'].value_counts()
Out[64]:
Scaling and Train Test Split¶
In [65]:
X = df.drop('price',axis=1)
y = df['price']
In [66]:
from sklearn.model_selection import train_test_split
In [67]:
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=101)
Scaling¶
In [68]:
from sklearn.preprocessing import MinMaxScaler
In [69]:
scaler = MinMaxScaler()
In [70]:
X_train= scaler.fit_transform(X_train)
In [71]:
X_test = scaler.transform(X_test)
In [72]:
X_train.shape
Out[72]:
In [73]:
X_test.shape
Out[73]:
Creating a Model¶
In [74]:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam
In [77]:
model = Sequential()
model.add(Dense(19, activation='relu'))
model.add(Dense(19, activation='relu'))
model.add(Dense(19, activation='relu'))
model.add(Dense(19, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
In [80]:
model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=128, epochs=400);
Evaluation on Test Data¶
https://scikit-learn.org/stable/modules/model_evaluation.html#regression-metrics
In [82]:
from sklearn.metrics import mean_squared_error,mean_absolute_error,explained_variance_score
Predicting on Brand New Data¶
In [81]:
X_test
Out[81]:
In [83]:
predictions = model.predict(X_test)
In [84]:
mean_absolute_error(y_test,predictions)
Out[84]:
In [85]:
np.sqrt(mean_squared_error(y_test,predictions))
Out[85]:
In [86]:
explained_variance_score(y_test,predictions)
Out[86]:
In [87]:
df['price'].mean()
Out[87]:
In [88]:
df['price'].median()
Out[88]:
In [89]:
# Our predictions
plt.scatter(y_test,predictions)
# Perfect predictions
plt.plot(y_test,y_test,'r')
Out[89]:
In [ ]:
errors = y_test.values.reshape(6480, 1) - predictions
In [91]:
sns.distplot(errors)
Predicting on a brand new house¶
In [92]:
single_house = df.drop('price',axis=1).iloc[0]
In [93]:
single_house = scaler.transform(single_house.values.reshape(-1, 19))
In [94]:
single_house
Out[94]:
In [95]:
model.predict(single_house)
Out[95]:
In [96]:
df.iloc[0]
Out[96]:
Keras TF 2.0 - Code Along Classification Project¶
Let's explore a classification task with Keras API for TF 2.0
The Data¶
Breast cancer wisconsin (diagnostic) dataset¶
Data Set Characteristics:
:Number of Instances: 569
:Number of Attributes: 30 numeric, predictive attributes and the class
:Attribute Information:
- radius (mean of distances from center to points on the perimeter)
- texture (standard deviation of gray-scale values)
- perimeter
- area
- smoothness (local variation in radius lengths)
- compactness (perimeter^2 / area - 1.0)
- concavity (severity of concave portions of the contour)
- concave points (number of concave portions of the contour)
- symmetry
- fractal dimension ("coastline approximation" - 1)
The mean, standard error, and "worst" or largest (mean of the three
largest values) of these features were computed for each image,
resulting in 30 features. For instance, field 3 is Mean Radius, field
13 is Radius SE, field 23 is Worst Radius.
- class:
- WDBC-Malignant
- WDBC-Benign
:Summary Statistics:
===================================== ====== ======
Min Max
===================================== ====== ======
radius (mean): 6.981 28.11
texture (mean): 9.71 39.28
perimeter (mean): 43.79 188.5
area (mean): 143.5 2501.0
smoothness (mean): 0.053 0.163
compactness (mean): 0.019 0.345
concavity (mean): 0.0 0.427
concave points (mean): 0.0 0.201
symmetry (mean): 0.106 0.304
fractal dimension (mean): 0.05 0.097
radius (standard error): 0.112 2.873
texture (standard error): 0.36 4.885
perimeter (standard error): 0.757 21.98
area (standard error): 6.802 542.2
smoothness (standard error): 0.002 0.031
compactness (standard error): 0.002 0.135
concavity (standard error): 0.0 0.396
concave points (standard error): 0.0 0.053
symmetry (standard error): 0.008 0.079
fractal dimension (standard error): 0.001 0.03
radius (worst): 7.93 36.04
texture (worst): 12.02 49.54
perimeter (worst): 50.41 251.2
area (worst): 185.2 4254.0
smoothness (worst): 0.071 0.223
compactness (worst): 0.027 1.058
concavity (worst): 0.0 1.252
concave points (worst): 0.0 0.291
symmetry (worst): 0.156 0.664
fractal dimension (worst): 0.055 0.208
===================================== ====== ======
:Missing Attribute Values: None
:Class Distribution: 212 - Malignant, 357 - Benign
:Creator: Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian
:Donor: Nick Street
:Date: November, 1995
This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets. https://goo.gl/U2Uwz2
Features are computed from a digitized image of a fine needle aspirate (FNA) of a breast mass. They describe characteristics of the cell nuclei present in the image.
Separating plane described above was obtained using Multisurface Method-Tree (MSM-T) [K. P. Bennett, "Decision Tree Construction Via Linear Programming." Proceedings of the 4th Midwest Artificial Intelligence and Cognitive Science Society, pp. 97-101, 1992], a classification method which uses linear programming to construct a decision tree. Relevant features were selected using an exhaustive search in the space of 1-4 features and 1-3 separating planes.
The actual linear program used to obtain the separating plane in the 3-dimensional space is that described in: [K. P. Bennett and O. L. Mangasarian: "Robust Linear Programming Discrimination of Two Linearly Inseparable Sets", Optimization Methods and Software 1, 1992, 23-34].
This database is also available through the UW CS ftp server:
ftp ftp.cs.wisc.edu cd math-prog/cpo-dataset/machine-learn/WDBC/
.. topic:: References
- W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction for breast tumor diagnosis. IS&T/SPIE 1993 International Symposium on Electronic Imaging: Science and Technology, volume 1905, pages 861-870, San Jose, CA, 1993.
- O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and prognosis via linear programming. Operations Research, 43(4), pages 570-577, July-August 1995.
- W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994) 163-171.
In [57]:
df = pd.read_csv('../DATA/cancer_classification.csv')
In [58]:
df.info()
In [59]:
df.describe().transpose()
Out[59]:
EDA¶
In [60]:
import seaborn as sns
import matplotlib.pyplot as plt
In [62]:
sns.countplot(x='benign_0__mal_1',data=df)
Out[62]:
In [63]:
sns.heatmap(df.corr())
Out[63]:
In [66]:
df.corr()['benign_0__mal_1'].sort_values()
Out[66]:
In [68]:
df.corr()['benign_0__mal_1'].sort_values().plot(kind='bar')
Out[68]:
In [70]:
df.corr()['benign_0__mal_1'][:-1].sort_values().plot(kind='bar')
Out[70]:
Train Test Split¶
In [73]:
X = df.drop('benign_0__mal_1',axis=1).values
y = df['benign_0__mal_1'].values
In [74]:
from sklearn.model_selection import train_test_split
In [76]:
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.25,random_state=101)
Scaling Data¶
In [77]:
from sklearn.preprocessing import MinMaxScaler
In [78]:
scaler = MinMaxScaler()
In [79]:
scaler.fit(X_train)
Out[79]:
In [80]:
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
Creating the Model¶
# For a binary classification problem
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])In [98]:
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation,Dropout
In [99]:
X_train.shape
Out[99]:
In [111]:
model = Sequential()
# https://stats.stackexchange.com/questions/181/how-to-choose-the-number-of-hidden-layers-and-nodes-in-a-feedforward-neural-netw
model.add(Dense(units=30,activation='relu'))
model.add(Dense(units=15,activation='relu'))
model.add(Dense(units=1,activation='sigmoid'))
# For a binary classification problem
model.compile(loss='binary_crossentropy', optimizer='adam')
In [112]:
# https://stats.stackexchange.com/questions/164876/tradeoff-batch-size-vs-number-of-iterations-to-train-a-neural-network
# https://datascience.stackexchange.com/questions/18414/are-there-any-rules-for-choosing-the-size-of-a-mini-batch
model.fit(x=X_train,
y=y_train,
epochs=600,
validation_data=(X_test, y_test), verbose=1
)
Out[112]:
In [113]:
# model.history.history
In [114]:
model_loss = pd.DataFrame(model.history.history)
In [115]:
# model_loss
In [116]:
model_loss.plot()
Out[116]:
Example Two: Early Stopping¶
We obviously trained too much! Let's use early stopping to track the val_loss and stop training once it begins increasing too much!
In [117]:
model = Sequential()
model.add(Dense(units=30,activation='relu'))
model.add(Dense(units=15,activation='relu'))
model.add(Dense(units=1,activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
In [119]:
from tensorflow.keras.callbacks import EarlyStopping
Stop training when a monitored quantity has stopped improving.
Arguments:
monitor: Quantity to be monitored.
min_delta: Minimum change in the monitored quantity
to qualify as an improvement, i.e. an absolute
change of less than min_delta, will count as no
improvement.
patience: Number of epochs with no improvement
after which training will be stopped.
verbose: verbosity mode.
mode: One of `{"auto", "min", "max"}`. In `min` mode,
training will stop when the quantity
monitored has stopped decreasing; in `max`
mode it will stop when the quantity
monitored has stopped increasing; in `auto`
mode, the direction is automatically inferred
from the name of the monitored quantity.In [121]:
early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
In [122]:
model.fit(x=X_train,
y=y_train,
epochs=600,
validation_data=(X_test, y_test), verbose=1,
callbacks=[early_stop]
)
Out[122]:
In [124]:
model_loss = pd.DataFrame(model.history.history)
model_loss.plot()
Out[124]:
Example Three: Adding in DropOut Layers¶
In [125]:
from tensorflow.keras.layers import Dropout
In [126]:
model = Sequential()
model.add(Dense(units=30,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(units=15,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(units=1,activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
In [127]:
model.fit(x=X_train,
y=y_train,
epochs=600,
validation_data=(X_test, y_test), verbose=1,
callbacks=[early_stop]
)
Out[127]:
In [128]:
model_loss = pd.DataFrame(model.history.history)
model_loss.plot()
Out[128]:
Model Evaluation¶
In [129]:
predictions = model.predict_classes(X_test)
In [130]:
from sklearn.metrics import classification_report,confusion_matrix
In [131]:
# https://en.wikipedia.org/wiki/Precision_and_recall
print(classification_report(y_test,predictions))
In [133]:
print(confusion_matrix(y_test,predictions))
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [1]:
n = int(input())
sv = list(map(int, input().split()))
mc = list(map(int, input().split()))
sv.sort()
mc.sort()
i = 0
while i < n:
if mc[i] < sv[i]:
i += 1
continue
else:
print('NO')
break
else:
print('YES')
In [5]:
try:
s = int(input())
d = int(input())
mn, mx = (10**(d-1)), (10**d - 1)
def sm(n):
r = 0
while n:
r, n = r + n % 10, n // 10
return r
for i in range(mn, mx+1):
if sm(i) == s:
print('Smallest number is',i)
break
else:
print('Not possible')
except:
print('Not possible')
In [10]:
n = int(input())
d = list(map(int, input().split()))
def checkt(degree, n):
smd = sum(degree)
if (2*(n-1) == smd):
return True
else:
return False
if checkt(d):
print('Yes')
else:
print('No')
In [2]:
#!/usr/bin/env python
# coding: utf-8
# In[103]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import find_peaks
import seaborn as sns
get_ipython().run_line_magic('matplotlib', 'inline')
# In[104]:
df=pd.read_csv('1 (2).csv')
# In[105]:
df.head()
# In[106]:
x=df['Time'][1:]
y=df['Average(A)'][1:]
# In[107]:
x.dtype
# In[108]:
y.dtype
# In[109]:
x = x.astype(float, errors = 'raise')
y = y.astype(float, errors = 'raise')
# In[110]:
sns.lineplot(x,y)
# In[111]:
peaks= find_peaks(y, height = 1,prominence=3, distance = 1)
height = peaks[1]['peak_heights'] #list of the heights of the peaks
peak_pos = x[peaks[0]] #list of the peaks positions
# In[112]:
print(height)
# In[113]:
print(peaks[0])
# In[114]:
print(peak_pos)
# In[118]:
plt.plot(peaks[0], y[peaks[0]], "xr")
plt.plot(y)
# plt.legend(['prominence'])
# In[119]:
from astropy import modeling
# In[147]:
fitter = modeling.fitting.LevMarLSQFitter()
model = modeling.models.Gaussian1D(amplitude=200, mean=0.026846, stddev=0.783768)
# depending on the data you need to give some initial values
fitted_model = fitter(model, x, y)
# In[148]:
plt.plot(x, y)
plt.plot(x, fitted_model(x))
# In[ ]:
Out[2]:
In [ ]:
In [ ]:
In [ ]: